package com.nlp.visualization.crawler.processor;

import com.nlp.visualization.common.CONSTANTS;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.JsonPathSelector;

import java.text.ParseException;
import java.util.Date;
import java.util.List;

/**
 * @author LXM
 * @Title: ${file_name}
 * @Description: ${todo}
 * @date 2018/3/3下午9:51
 */
public class ProcessorNeteasy extends Processor {

    private static final String NETEASY_URL = CONSTANTS.NETEASY_URL;

    //定义如何抽取网易新闻页面的信息
    @Override
    public void processor(Page page) {

        if (page.getUrl().toString().equals(NETEASY_URL))
            initProcessor(page);
        else
            contentProcessor(page);

    }

    @Override
    public void processor(Page page, String date) {

    }


    @Override
    public void init(Spider spider) {

        spider.addRequest(new Request(NETEASY_URL).putExtra("_charset", "gb2312").setPriority(1));

    }

    /**
     * 保存要爬取的链接的
     *
     * @param page
     */
    private void initProcessor(Page page) {
        List<String> linkList = new JsonPathSelector("$.news[*]").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        List<String> tags = new JsonPathSelector("$.category[*].n").selectList(page.getRawText().substring(page.getRawText().indexOf("{")));
        int count = 0;
        for (int i = 0; i < linkList.size(); i++) {
            List<String> link = new JsonPathSelector("$[*].l").selectList(linkList.get(i));
            List<String> title = new JsonPathSelector("$[*].t").selectList(linkList.get(i));
            List<String> time = new JsonPathSelector("$[*].p").selectList(linkList.get(i));
            String tag = tags.get(i);
            count += link.size();
            for (int j = 0; j < link.size(); j++) {
                if (strToDate(time.get(j)).after(today()))
                    page.addTargetRequest(new Request(link.get(j)).putExtra("title", title.get(j)).putExtra("time", strToDate(time.get(j))).putExtra("tag", tag));
            }
        }
        logger.debug(String.format("【%s】%d条记录加入任务队列", getSourceFromPage(page), count));
        page.setSkip(true);
    }

    /**
     * 用于保存页面元素的
     *
     * @param page
     */
    private void contentProcessor(Page page) {
        Html html = page.getHtml();
        //保存内容正文
        page.putField("article", html.$("article").toString());
        //保存正文标题
        page.putField("main-title", html.$("main-title").toString());
        //保存URL
        page.putField("url", page.getUrl());
        //保存发表时间
        String timeStr = html.$("date").toString();
        page.putField("date", timeStr);
    }

    private static Date today() {
        String strTime = sdf.format(new Date());
        strTime = strTime.substring(0, 10);
        strTime = strTime + " 00:00:00";
        Date today = new Date();
        try {
            today = sdf.parse(strTime);
        } catch (ParseException e1) {
            e1.printStackTrace();
        }            //今天零点的时间
        return today;
    }

    private static Date strToDate(String str) {
        Date newsTime = new Date();
        try {
            newsTime = sdf.parse(str);
        } catch (ParseException e1) {
            e1.printStackTrace();
        }
        return newsTime;
    }

}
